{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import random\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义读取数据的类,像mnist那样\n",
    "class WSTrainData(object):\n",
    "    def __init__(self, data_folder):\n",
    "        self.data_folder = data_folder\n",
    "        self.features = []\n",
    "        self.labels = []\n",
    "        \n",
    "        self._query_data()\n",
    "        print(\"features: %d, labels: %d\" % (len(self.features), len(self.labels)))\n",
    "    \n",
    "    def _query_data(self):\n",
    "        print(self.data_folder)\n",
    "        for root, dirs, files in os.walk(self.data_folder):\n",
    "            for ifile in files:\n",
    "                cur_file = os.path.join(root, ifile)\n",
    "                cur_features, cur_labels = self._query_file_data(cur_file)\n",
    "                self.features.extend(cur_features)\n",
    "                self.labels.extend(cur_labels)\n",
    "                \n",
    "    def _query_file_data(self, file_name):\n",
    "        features = []\n",
    "        labels = []\n",
    "        with open(file_name, 'r', encoding=\"utf-8-sig\") as pfile:\n",
    "            for isent in [x.strip() for x in pfile if x.strip() != \"\"]:\n",
    "                words = isent.split()\n",
    "                chars = []\n",
    "                tags = []\n",
    "                for iword in words:\n",
    "                    chars.extend(list(iword))\n",
    "                    if len(iword) == 1:\n",
    "                        tags.append(\"S\")\n",
    "                    else:\n",
    "                        tags.extend([\"B\"]+[\"M\"]*(len(iword)-2)+[\"E\"])\n",
    "                features.append(chars)\n",
    "                labels.append(tags)\n",
    "        \n",
    "        return features, labels\n",
    "    \n",
    "    def next_batch(self, batch_size):\n",
    "        return list(zip(*random.sample(list(zip(self.features, self.labels)), batch_size)))\n",
    "    \n",
    "    def get_data(self):\n",
    "        return self.features, self.labels\n",
    "    \n",
    "\n",
    "# class WSDataReader(object):\n",
    "#     def __init__(self, train_folder, test_folder, dev_folder):\n",
    "#         self.train_data = "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 创建数据转换等工具函数\n",
    "def create_dic(item_list, add_unk=False, add_pad=False):\n",
    "    \"\"\"\n",
    "    Create a dictionary of items from a list of list of items.\n",
    "    \"\"\"\n",
    "    assert type(item_list) in (list, tuple)\n",
    "    dic = {}\n",
    "    for items in item_list:\n",
    "        for item in items:\n",
    "            if item not in dic:\n",
    "                dic[item] = 1\n",
    "            else:\n",
    "                dic[item] += 1\n",
    "    # Make sure that <PAD> have a id 0.\n",
    "    if add_pad:\n",
    "        dic['<PAD>'] = 1e20\n",
    "    # If specified, add a special item <UNK>.\n",
    "    if add_unk:\n",
    "        dic['<UNK>'] = 1e10\n",
    "    return dic\n",
    "\n",
    "def create_mapping(items):\n",
    "    \"\"\"\n",
    "    Create a mapping (item to ID / ID to item) from a dictionary.\n",
    "    Items are ordered by decreasing frequency.\n",
    "    \"\"\"\n",
    "    if type(items) is dict:\n",
    "        sorted_items = sorted(items.items(), key=lambda x: (-x[1], x[0]))\n",
    "        id2item = {i: v[0] for i, v in enumerate(sorted_items)}\n",
    "        item2id = {v: k for k, v in id2item.items()}\n",
    "        return item2id, id2item\n",
    "    elif type(items) is list:\n",
    "        id2item = {i: v for i, v in enumerate(items)}\n",
    "        item2id = {v: k for k, v in id2item.items()}\n",
    "        return item2id, id2item\n",
    "\n",
    "\n",
    "def create_input(batch):\n",
    "    \"\"\"\n",
    "    Take each sentence data in batch and return an input for\n",
    "    the training or the evaluation function.\n",
    "    \"\"\"\n",
    "    assert len(batch) > 0\n",
    "    lengths = [len(seq) for seq in batch]\n",
    "    max_len = max(2, max(lengths))\n",
    "    ret = []\n",
    "    for seq_id, pos in zip(batch, lengths):\n",
    "        assert len(seq_id) == pos\n",
    "        pad = [0] * (max_len - pos)\n",
    "        ret.append(np.array(seq_id + pad))\n",
    "    ret.append(lengths)\n",
    "    return ret\n",
    "\n",
    "def data_to_ids(data, mappings):\n",
    "    \"\"\"\n",
    "    Map text data to ids.\n",
    "    \"\"\"\n",
    "\n",
    "    def strQ2B(ustring):\n",
    "        rstring = \"\"\n",
    "        for uchar in ustring:\n",
    "            inside_code = ord(uchar)\n",
    "            if inside_code == 12288:\n",
    "                inside_code = 32\n",
    "            elif 65281 <= inside_code <= 65374:\n",
    "                inside_code -= 65248\n",
    "            rstring += chr(inside_code)\n",
    "        return rstring\n",
    "    def strB2Q(ustring):\n",
    "        rstring = \"\"\n",
    "        for uchar in ustring:\n",
    "            inside_code = ord(uchar)\n",
    "            if inside_code == 32:\n",
    "                inside_code = 12288\n",
    "            elif 32 <= inside_code <= 126:\n",
    "                inside_code += 65248\n",
    "            rstring += chr(inside_code)\n",
    "        return rstring\n",
    "\n",
    "    def map(item, mapping):\n",
    "        if item in mapping:\n",
    "            return mapping[item]\n",
    "        item = strB2Q(item)\n",
    "        if item in mapping:\n",
    "            return mapping[item]\n",
    "        item = strQ2B(item)\n",
    "\n",
    "    def map_seq(seqs, mapping):\n",
    "        return [[map(item, mapping) for item in seq] for seq in seqs]\n",
    "\n",
    "    ret = []\n",
    "    for d, m in zip(data, mappings):\n",
    "        ret.append(map_seq(d, m))\n",
    "    return tuple(ret)\n",
    "\n",
    "def data_iterator(inputs, batch_size, shuffle=True, max_length=200):\n",
    "    \"\"\"\n",
    "    A simple iterator for generating dynamic mini batches.\n",
    "    \"\"\"\n",
    "    assert len(inputs) > 0\n",
    "    assert all([len(item) == len(inputs[0]) for item in inputs])\n",
    "    inputs = zip(*inputs)\n",
    "    if shuffle:\n",
    "        np.random.shuffle(inputs)\n",
    "\n",
    "    batch = []\n",
    "    bs = batch_size\n",
    "    for d in inputs:\n",
    "        if len(d[0]) > max_length:\n",
    "            bs = max(1, min(batch_size * max_length / len(d[0]), bs))\n",
    "        if len(batch) < bs:\n",
    "            batch.append(d)\n",
    "        else:\n",
    "            yield zip(*batch)\n",
    "            batch = [d]\n",
    "            if len(d[0]) < max_length:\n",
    "                bs = batch_size\n",
    "            else:\n",
    "                bs = max(1, batch_size * max_length / len(d[0]))\n",
    "    if batch:\n",
    "        yield zip(*batch)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./datasets/sighan2005-pku/\n",
      "features: 0, labels: 0\n"
     ]
    }
   ],
   "source": [
    "train_data = WSTrainData(\"./datasets/sighan2005-pku/\")\n",
    "# 创建训练数据的字典,生成mapping,然后把训练数据都转为ID\n",
    "train_features_list, train_label_list = train_data.get_data()\n",
    "\n",
    "feature_dict = create_dic(train_features_list)\n",
    "label_dict = create_dic(train_label_list)\n",
    "\n",
    "feature_mapping = create_mapping(feature_dict)\n",
    "label_mapping = create_mapping(label_dict)\n",
    "\n",
    "feature_ids_list = data_to_ids(train_features_list, feature_mapping)\n",
    "label_ids_list = data_to_ids(train_label_list, label_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./data/data/train_pku\n",
      "features: 17149, labels: 17149\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0807 14:53:30.708232 140377277859648 deprecation.py:506] From /home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0807 14:53:30.723998 140377277859648 deprecation.py:506] From <ipython-input-5-4c3558fbcb6f>:24: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0807 14:53:31.178775 140377277859648 lazy_loader.py:50] \n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "W0807 14:53:31.396484 140377277859648 deprecation.py:323] From <ipython-input-5-4c3558fbcb6f>:40: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost: 1.061541, train accuracy: 0.864049\n",
      "cost: 0.978554, train accuracy: 0.875934\n",
      "cost: 0.829364, train accuracy: 0.872427\n",
      "cost: 0.742704, train accuracy: 0.889205\n",
      "cost: 0.651568, train accuracy: 0.899818\n",
      "cost: 0.592077, train accuracy: 0.886831\n",
      "cost: 0.492185, train accuracy: 0.914637\n",
      "cost: 0.466398, train accuracy: 0.899110\n",
      "cost: 0.425192, train accuracy: 0.920716\n"
     ]
    }
   ],
   "source": [
    "# 数据处理完毕,接下来,定义模型结构:前向网络,损失函数,优化器,精度计算,运行网络\n",
    "num_classes = 4 # B,E,M,S\n",
    "\n",
    "# embedding 网络\n",
    "def build_input_graph(vocab_size, emb_size):\n",
    "    \"\"\"\n",
    "    transform to embeddings from lookup tables\n",
    "    \"\"\"\n",
    "    x = tf.placeholder(dtype=tf.int32, shape=[None, None])\n",
    "    y = tf.placeholder(dtype = tf.int32, shape=[None, None])\n",
    "    \n",
    "    embeddings = tf.get_variable(\"embeddings\", [vocab_size, emb_size])\n",
    "    embedding_output = tf.nn.embedding_lookup(embeddings, x)\n",
    "    \n",
    "    lengths = tf.placeholder(dtype=tf.int32, shape=[None])\n",
    "    \n",
    "    return x, y, embeddings, embedding_output, lengths\n",
    "\n",
    "# 简单的一层卷积，一层fc。\n",
    "def build_tag_graph(inputs, kernal_size, chanel_num, num_classes, cur_label_ids_padding, lengths, embedding_size=256):\n",
    "    # 创建卷积层\n",
    "    # 定义一个卷积核和偏置项\n",
    "    fitler_w = tf.get_variable(\"kernal_weight_matrix\", shape=[kernal_size, embedding_size, chanel_num], dtype=tf.float32)\n",
    "    bias = tf.get_variable(\"bias\", shape=[chanel_num], dtype=tf.float32, initializer=tf.zeros_initializer(tf.float32))\n",
    "    \n",
    "    conv_output = tf.nn.conv1d(inputs, fitler_w, 1, \"SAME\") + bias\n",
    "    # mask掉padding\n",
    "    masks = tf.cast(tf.sequence_mask(lengths), tf.float32)\n",
    "    conv_output = tf.multiply(conv_output, tf.expand_dims(masks, -1))\n",
    "    \n",
    "    # fc层\n",
    "    fc_output = tf.contrib.layers.fully_connected(conv_output, num_classes, tf.identity)\n",
    "    \n",
    "    final_output = tf.reshape(fc_output, [-1, num_classes])\n",
    "    \n",
    "    cur_label_one_hot = tf.one_hot(cur_label_ids_padding, 4)\n",
    "    y_real = tf.reshape(cur_label_one_hot, [-1, num_classes])\n",
    "\n",
    "    # 计算损失\n",
    "    cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=final_output, labels=y_real))\n",
    "    \n",
    "    # 计算当前训练集上的精度\n",
    "    correct_pred = tf.equal(tf.argmax(final_output, 1), tf.argmax(y_real, 1))\n",
    "    \n",
    "    all_chars_num = tf.cast(tf.shape(tf.reshape(inputs, [-1, 256]))[0], tf.float32)\n",
    "    all_true_chars_num = tf.cast(tf.reduce_sum(lengths), tf.float32)\n",
    "    accuracy = (tf.reduce_sum(tf.cast(correct_pred, tf.float32))+all_true_chars_num-all_chars_num)/all_true_chars_num\n",
    "    \n",
    "    return cross_entropy, accuracy\n",
    "\n",
    "\n",
    "def train(data_folder):\n",
    "    # 获取数据\n",
    "    all_data_obj = WSTrainData(data_folder)\n",
    "    # 创建dict和mapping，并将数据转为id序列\n",
    "    feature_dic = create_dic(all_data_obj.features, True, True)\n",
    "    label_dic = create_dic(all_data_obj.labels)\n",
    "    feature_2_id, id_2_feature = create_mapping(feature_dic)\n",
    "    label_2_id, id_2_label = create_mapping(label_dic)\n",
    "    \n",
    "    # 输出batch进行训练\n",
    "    iters = 40000\n",
    "    batch_size = 32\n",
    "    embedding_size = 256\n",
    "    # 创建网络\n",
    "    x, y, embeddings, embeddings_out, lengths = build_input_graph(len(feature_dic), embedding_size)\n",
    "    cross_entropy, accuracy = build_tag_graph(embeddings_out, 512, 128, 4, y, lengths)\n",
    "    \n",
    "    # 创建优化方案\n",
    "    lr = 1e-3\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(cross_entropy)\n",
    "    \n",
    "    # 创建session并且初始化\n",
    "    init_op = tf.global_variables_initializer() \n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        sess.run(init_op)\n",
    "        for i in range(iters):\n",
    "            cur_features, cur_labels = all_data_obj.next_batch(batch_size)\n",
    "#             print(cur_features)\n",
    "#             print(len(cur_features))\n",
    "            cur_feature_ids = data_to_ids([cur_features], mappings=[feature_2_id])[0]\n",
    "#             print(np.array(cur_feature_ids).shape)\n",
    "            cur_label_ids = data_to_ids([cur_labels], mappings=[label_2_id])[0]\n",
    "            # padding\n",
    "            input_info = create_input(cur_feature_ids)\n",
    "            cur_feature_ids_padding = input_info[:-1]\n",
    "            cur_lengths = input_info[-1]\n",
    "            \n",
    "            label_info = create_input(cur_label_ids)\n",
    "            cur_label_ids_padding = label_info[:-1]\n",
    "            cur_label_lengths = label_info[-1]\n",
    "            assert cur_lengths == cur_label_lengths\n",
    "            \n",
    "            sess.run(optimizer, feed_dict={x: cur_feature_ids_padding, y: cur_label_ids_padding, lengths: cur_lengths})\n",
    "            if (i+1)%100 == 0:\n",
    "                cost, cur_accuracy = sess.run([cross_entropy, accuracy], feed_dict={x: cur_feature_ids_padding, y: cur_label_ids_padding, lengths: cur_lengths})\n",
    "                print(\"cost: %f, train accuracy: %f\" % (cost, cur_accuracy))\n",
    "                \n",
    "train_data_folder = \"./data/data/train_pku\"\n",
    "train(train_data_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mx_py3",
   "language": "python",
   "name": "mx_py3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
